
# General imports
from pathlib import Path
import os
import numpy as np
import pandas as pd
from tqdm.notebook import tqdm
from datetime import datetime
import time
import json

import warnings
warnings.filterwarnings('ignore')

# Pytorch imports
import gym
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchsummary import summary

# Plotting imports
import matplotlib.pyplot as plt
import seaborn as sns

# Custom imports
from utils.plot import plot_results_hist, plot_training_results, plot_test_episodes, plot_n_training, compare_training_hp
from agents.dqn import Qnet, DQNAgent
from agents.random import randAgent
from utils.utils import play_episode, play_n_episodes


print(f'Installed GYM version is: {gym.__version__}')

Installed GYM version is: 0.26.2


# Set Seaborn aesthetics
sns.set_style('darkgrid')
plt.rcParams['figure.figsize'] = (6,4)
plt.rcParams['lines.linewidth'] = 2
plt.rcParams['axes.titlesize'] = 14
plt.rcParams['axes.titlelocation'] = 'left'
plt.style.use('ggplot')


env = gym.make("LunarLander-v2", render_mode='rgb_array')
print(f'Reward threshold is: {env.spec.reward_threshold}')
print(f'Reward range is: {env.reward_range}')
print(f'Maximun number of steps per episode: {env.spec.max_episode_steps}')
print(f'Action space dimension is: {env.action_space.n}')
print(f'Observation space dimension is: {env.observation_space.shape}')

Reward threshold is: 200
Reward range is: (-inf, inf)
Maximun number of steps per episode: 1000
Action space dimension is: 4
Observation space dimension is: (8,)


random_agent = randAgent(env)
total_reward, steps = play_episode(env, random_agent, save=True)
print(f'Total reward: {total_reward}, Steps: {steps}')

Total reward: -213.00483424337665, Steps: 112


n_episodes = 1000
total_rewards, n_steps = play_n_episodes(env, random_agent, n_episodes)
# Plotting the results
plot_results_hist(n_steps, total_rewards)

  0%|          | 0/1000 [00:00<?, ?it/s]


############################
## SET DEFAULT PARAMETERS ##
############################

# Parameters that we are not going to change
MAX_EPISODES = 1000   # Maximun number of episodes
MIN_EPISODES = 250    # Minimun number of episodes
EPSILON_START = 1     # Initial value of epsilon
EPSILON_DECAY = .995  # Decay rate of epsilon
EPSILON_END = 0.01    # Final value of epsilon
BURN_IN = 1000        # Number of steps to fill the buffer

# Parameters that can be changed (hyperparameters)
GAMMA = 0.99          # Gamma parameter for the Bellman equation
BATCH_SIZE = 32       # Batch size
LR = 0.001           # Learning rate for the optimizer
MEMORY_SIZE = 8000    # Maximum size of the replay buffer
DNN_UPD = 4           # Frequency of updating the main network
DNN_SYNC = 100        # Frequency of synchronizing the target network
N_NEURONS = 64       # Number of neurons in the hidden layers

# Parameters to change agent's topology (DQN, Double DQN, Dueling DQN, NoisyNet DQN or any combination of them)
NOISY_NET = False      # Use noisy network
DUELING = False         # Use dueling network
DOUBLE = False         # Use double network


env = gym.make('LunarLander-v2')
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
net = Qnet(n_inputs=env.observation_space.shape[0], n_hidden=N_NEURONS, 
             n_outputs=env.action_space.n).to(device)
obs = env.reset()[0]
obs_tensor = torch.Tensor(obs).to(device).unsqueeze(0)
print(f'Input tensor to the NN: {obs_tensor.shape}')
output = net(obs_tensor)
print(f'Output tensor:{output}')
print(f'Output tensor shape:{output.shape}')

Input tensor to the NN: torch.Size([1, 8])
Output tensor:tensor([[ 0.1292, -0.0257, -0.1394, -0.0923]], device='cuda:0',
       grad_fn=<AddmmBackward0>)
Output tensor shape:torch.Size([1, 4])


print('Network architecture:')
summary(net, input_size=(1,net.n_inputs))

Network architecture:
----------------------------------------------------------------
        Layer (type)               Output Shape         Param #
================================================================
            Linear-1                [-1, 1, 64]             576
            Linear-2                [-1, 1, 64]           4,160
            Linear-3                 [-1, 1, 4]             260
================================================================
Total params: 4,996
Trainable params: 4,996
Non-trainable params: 0
----------------------------------------------------------------
Input size (MB): 0.00
Forward/backward pass size (MB): 0.00
Params size (MB): 0.02
Estimated Total Size (MB): 0.02
----------------------------------------------------------------


def train_n(params, n_iter):
    env = gym.make("LunarLander-v2", render_mode='rgb_array')
    #device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    device = torch.device('cpu')
    n_training_res = []
    for i in range(n_iter):
        agent = DQNAgent(env, device=device, n_hidden=params['N_NEURONS'], mem_size=params['MEMORY_SIZE'],
                         gamma=params['GAMMA'], lr=params['LR'], noisyNet=params['NOISY_NET'], 
                         dueling=params['DUELING'], double=params['DOUBLE'])

        # Train agent
        res = agent.train(max_episodes=MAX_EPISODES, 
                                    min_episodios=MIN_EPISODES, 
                                    batch_size=params['BATCH_SIZE'], 
                                    eps_start=EPSILON_START, 
                                    eps_end=EPSILON_END,
                                    eps_decay=EPSILON_DECAY,
                                    dnn_update_frequency=params['DNN_UPD'], 
                                    dnn_sync_frequency=params['DNN_SYNC'])
        n_training_res.append(res)
        del agent
        torch.cuda.empty_cache()
    return n_training_res


dqn_hyperparams = {
    'DUELING': False,
    'NOISY_NET': False,
    'DOUBLE': False,
    'N_NEURONS': N_NEURONS,
    'BATCH_SIZE': BATCH_SIZE,
    'GAMMA': GAMMA,
    'LR': LR,
    'MEMORY_SIZE': MEMORY_SIZE,
    'DNN_SYNC': DNN_SYNC,
    'DNN_UPD': DNN_UPD,
}

dqn_init_res = train_n(dqn_hyperparams, 1)

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 455 Mean Rewards 200.01 Epsilon 0.10		Environment solved in 455 episodes!


plot_training_results(dqn_init_res[0], 200)


def tune_hyperparameter(default_params, param_name, values, n_iter):
    result = {}
    params = default_params.copy()
    for value in values:
        params[param_name] = value
        res = train_n(params, n_iter)
        result[value] = res
    return result


dqn_hyperparams = {
    'DUELING': False,
    'NOISY_NET': False,
    'DOUBLE': False,
    'N_NEURONS': N_NEURONS,
    'BATCH_SIZE': BATCH_SIZE,
    'GAMMA': GAMMA,
    'LR': LR,
    'MEMORY_SIZE': MEMORY_SIZE,
    'DNN_SYNC': DNN_SYNC,
    'DNN_UPD': DNN_UPD,
}

tune_params = {
    'LR': [0.01, 0.005, 0.001, 0.0005, 0.0001],
    'BATCH_SIZE': [32, 64, 128],
    'N_NEURONS': [32, 64, 128],
    'DNN_UPD': [2, 4, 8],
    'DNN_SYNC': [100, 500, 1000],
    'MEMORY_SIZE': [4000, 8000, 16000, 32000],
    'GAMMA': [0.97, 0.99, 0.999]
    }

res_dqn_hp = {}

for param_name, param_values in tune_params.items():
    res_dqn_hp[param_name] = tune_hyperparameter(dqn_hyperparams, param_name, param_values, 5)

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 1000 Mean Rewards -383.52 Epsilon 0.01		Episode limit reached.

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 1000 Mean Rewards -357.38 Epsilon 0.01		Episode limit reached.

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 1000 Mean Rewards -348.67 Epsilon 0.01		Episode limit reached.

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 1000 Mean Rewards -377.55 Epsilon 0.01		Episode limit reached.

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 1000 Mean Rewards -223.60 Epsilon 0.01		Episode limit reached.


def json_default(obj):
    if isinstance(obj, np.ndarray):
        return obj.tolist()
    raise TypeError('Not serializable')

with open(Path('training_stats')/'dqn_hyperparams.json', 'w') as f:
    json.dump(res_dqn_hp, f, default=json_default)


with open(Path('training_stats')/'dqn_hyperparams.json') as f:
    dqn_hyper = json.loads(f.read())
    
for param_name in dqn_hyper.keys():
    fig=compare_training_hp(dqn_hyper[param_name], 200, param_name)
    fig.savefig(Path('img')/f'dqn_param_{param_name}.png', dpi=300)


# Parameters that can be changed (hyperparameters)
GAMMA = 0.99          # Gamma parameter for the Bellman equation
BATCH_SIZE = 64       # Batch size
LR = 0.001           # Learning rate for the optimizer
MEMORY_SIZE = 8000    # Maximum size of the replay buffer
DNN_UPD = 2           # Frequency of updating the main network
DNN_SYNC = 500        # Frequency of synchronizing the target network
N_NEURONS = 128       # Number of neurons in the hidden layers

# Parameters to change agent's topology (DQN, Double DQN, Dueling DQN, NoisyNet DQN or any combination of them)
NOISY_NET = False      # Use noisy network
DUELING = False         # Use dueling network
DOUBLE = False         # Use double network


dqn_hyperparams = {
    'DUELING': False,
    'NOISY_NET': False,
    'DOUBLE': False,
    'N_NEURONS': N_NEURONS,
    'BATCH_SIZE': BATCH_SIZE,
    'GAMMA': GAMMA,
    'LR': LR,
    'MEMORY_SIZE': MEMORY_SIZE,
    'DNN_SYNC': DNN_SYNC,
    'DNN_UPD': DNN_UPD,
}

dqn_res = train_n(dqn_hyperparams, 10)

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 622 Mean Rewards 200.22 Epsilon 0.04		Environment solved in 622 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 457 Mean Rewards 200.13 Epsilon 0.10		Environment solved in 457 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 439 Mean Rewards 203.29 Epsilon 0.11		Environment solved in 439 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 716 Mean Rewards 200.09 Epsilon 0.03		Environment solved in 716 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 611 Mean Rewards 202.52 Epsilon 0.05		Environment solved in 611 episodes!


fig = plot_n_training(dqn_res, 200, 'DQN')
fig.savefig(Path('img')/f'dqn_opt_n_train.png', dpi=300)


params = {
    'DUELING': False,
    'NOISY_NET': False,
    'DOUBLE': False,
    'N_NEURONS': N_NEURONS,
    'BATCH_SIZE': BATCH_SIZE,
    'GAMMA': GAMMA,
    'LR': LR,
    'MEMORY_SIZE': MEMORY_SIZE,
    'BATCH_SIZE': BATCH_SIZE,
    'DNN_SYNC': DNN_SYNC,
    'DNN_UPD': DNN_UPD,
}

env = gym.make("LunarLander-v2", render_mode='rgb_array')

DQNagent = DQNAgent(env, device='cpu', n_hidden=params['N_NEURONS'], mem_size=params['MEMORY_SIZE'],
                    gamma=params['GAMMA'], lr=params['LR'], noisyNet=params['NOISY_NET'], 
                    dueling=params['DUELING'], double=params['DOUBLE'])

# Train agent
dqn_opt_res = DQNagent.train(max_episodes=MAX_EPISODES, 
                            min_episodios=MIN_EPISODES, 
                            batch_size=params['BATCH_SIZE'], 
                            eps_start=EPSILON_START, 
                            eps_end=EPSILON_END,
                            eps_decay=EPSILON_DECAY,
                            dnn_update_frequency=params['DNN_UPD'], 
                            dnn_sync_frequency=params['DNN_SYNC'])

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 693 Mean Rewards 201.53 Epsilon 0.03		Environment solved in 693 episodes!


total_reward_dqn, steps_dqn = play_episode(env, DQNagent, save=True)
print(f'Total reward: {total_reward_dqn}, Steps: {steps_dqn}')

Total reward: 247.55224297338785, Steps: 410


n_episodes = 1000
total_rewards_dqn, n_steps_dqn = play_n_episodes(env, DQNagent, n_episodes)
plot_results_hist(n_steps_dqn, total_rewards_dqn)

  0%|          | 0/1000 [00:00<?, ?it/s]


dqn_hyperparams = {
    'DUELING': False,
    'NOISY_NET': False,
    'DOUBLE': True,
    'N_NEURONS': N_NEURONS,
    'BATCH_SIZE': BATCH_SIZE,
    'GAMMA': GAMMA,
    'LR': LR,
    'MEMORY_SIZE': MEMORY_SIZE,
    'BATCH_SIZE': BATCH_SIZE,
    'DNN_SYNC': DNN_SYNC,
    'DNN_UPD': DNN_UPD,
}

dqn_double_res = train_n(dqn_hyperparams, 10)

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 496 Mean Rewards 200.78 Epsilon 0.08		Environment solved in 496 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 607 Mean Rewards 201.79 Epsilon 0.05		Environment solved in 607 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 458 Mean Rewards 200.58 Epsilon 0.10		Environment solved in 458 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 796 Mean Rewards 201.00 Epsilon 0.02		Environment solved in 796 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 442 Mean Rewards 201.26 Epsilon 0.11		Environment solved in 442 episodes!


plot_n_training(dqn_double_res, 200, 'Double DQN')


dqn_hyperparams = {
    'DUELING': False,
    'NOISY_NET': False,
    'DOUBLE': True,
    'N_NEURONS': N_NEURONS,
    'BATCH_SIZE': BATCH_SIZE,
    'GAMMA': GAMMA,
    'LR': 0.00025,
    'MEMORY_SIZE': MEMORY_SIZE,
    'BATCH_SIZE': BATCH_SIZE,
    'DNN_SYNC': DNN_SYNC,
    'DNN_UPD': DNN_UPD,
}

dqn_double_res2 = train_n(dqn_hyperparams, 10)

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 483 Mean Rewards 201.69 Epsilon 0.09		Environment solved in 483 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 424 Mean Rewards 201.54 Epsilon 0.12		Environment solved in 424 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 427 Mean Rewards 200.89 Epsilon 0.12		Environment solved in 427 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 487 Mean Rewards 202.51 Epsilon 0.09		Environment solved in 487 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 502 Mean Rewards 200.14 Epsilon 0.08		Environment solved in 502 episodes!


plot_n_training(dqn_double_res2, 200, 'Double DQN')


dqn_hyperparams = {
    'DUELING': True,
    'NOISY_NET': False,
    'DOUBLE': False,
    'N_NEURONS': N_NEURONS,
    'BATCH_SIZE': BATCH_SIZE,
    'GAMMA': GAMMA,
    'LR': LR,
    'MEMORY_SIZE': MEMORY_SIZE,
    'BATCH_SIZE': BATCH_SIZE,
    'DNN_SYNC': DNN_SYNC,
    'DNN_UPD': DNN_UPD,
}

dqn_dueling_res = train_n(dqn_hyperparams, 10)

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 488 Mean Rewards 202.44 Epsilon 0.09		Environment solved in 488 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 676 Mean Rewards 200.83 Epsilon 0.03		Environment solved in 676 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 564 Mean Rewards 204.38 Epsilon 0.06		Environment solved in 564 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 463 Mean Rewards 200.78 Epsilon 0.10		Environment solved in 463 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 428 Mean Rewards 201.14 Epsilon 0.12		Environment solved in 428 episodes!


plot_n_training(dqn_dueling_res, 200, 'Dueling DQN')


dqn_hyperparams = {
    'DUELING': False,
    'NOISY_NET': True,
    'DOUBLE': False,
    'N_NEURONS': N_NEURONS,
    'BATCH_SIZE': BATCH_SIZE,
    'GAMMA': GAMMA,
    'LR': LR,
    'MEMORY_SIZE': MEMORY_SIZE,
    'BATCH_SIZE': BATCH_SIZE,
    'DNN_SYNC': DNN_SYNC,
    'DNN_UPD': DNN_UPD,
}

dqn_noisy_res = train_n(dqn_hyperparams, 10)

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 491 Mean Rewards 201.39 Epsilon 0.09		Environment solved in 491 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 495 Mean Rewards 201.46 Epsilon 0.08		Environment solved in 495 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 654 Mean Rewards 201.35 Epsilon 0.04		Environment solved in 654 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 532 Mean Rewards 202.30 Epsilon 0.07		Environment solved in 532 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 779 Mean Rewards 202.12 Epsilon 0.02		Environment solved in 779 episodes!


plot_n_training(dqn_noisy_res, 200, 'Noisy DQN')


dqn_hyperparams = {
    'DUELING': True,
    'NOISY_NET': False,
    'DOUBLE': True,
    'N_NEURONS': N_NEURONS,
    'BATCH_SIZE': BATCH_SIZE,
    'GAMMA': GAMMA,
    'LR': LR,
    'MEMORY_SIZE': MEMORY_SIZE,
    'BATCH_SIZE': BATCH_SIZE,
    'DNN_SYNC': DNN_SYNC,
    'DNN_UPD': DNN_UPD,
}

dqn_double_dueling_res = train_n(dqn_hyperparams, 10)

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 595 Mean Rewards 204.23 Epsilon 0.05		Environment solved in 595 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 510 Mean Rewards 200.17 Epsilon 0.08		Environment solved in 510 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 501 Mean Rewards 200.16 Epsilon 0.08		Environment solved in 501 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 632 Mean Rewards 200.40 Epsilon 0.04		Environment solved in 632 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 421 Mean Rewards 201.65 Epsilon 0.12		Environment solved in 421 episodes!


plot_n_training(dqn_double_dueling_res, 200, 'Double + Dueling DQN')


dqn_hyperparams = {
    'DUELING': True,
    'NOISY_NET': False,
    'DOUBLE': True,
    'N_NEURONS': N_NEURONS,
    'BATCH_SIZE': BATCH_SIZE,
    'GAMMA': GAMMA,
    'LR': LR,
    'MEMORY_SIZE': MEMORY_SIZE,
    'BATCH_SIZE': BATCH_SIZE,
    'DNN_SYNC': DNN_SYNC,
    'DNN_UPD': DNN_UPD,
}

tune_params = {
    'LR': [0.001, 0.0005, 0.00025, 0.0001],
    'BATCH_SIZE': [32, 64, 128],
    'N_NEURONS': [32, 64, 128],
    'DNN_UPD': [1, 2, 4, 8],
    'MEMORY_SIZE': [4000, 8000, 16000, 32000],
    'GAMMA': [0.98, 0.99, 0.995]
    }

res = {}

for param_name, param_values in tune_params.items():
    res[param_name] = tune_hyperparameter(dqn_hyperparams, param_name, param_values, 5)

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 733 Mean Rewards 201.94 Epsilon 0.03		Environment solved in 733 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 551 Mean Rewards 200.32 Epsilon 0.06		Environment solved in 551 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 577 Mean Rewards 200.71 Epsilon 0.06		Environment solved in 577 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 689 Mean Rewards 202.61 Epsilon 0.03		Environment solved in 689 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 681 Mean Rewards 200.40 Epsilon 0.03		Environment solved in 681 episodes!


def json_default(obj):
    if isinstance(obj, np.ndarray):
        return obj.tolist()
    raise TypeError('Not serializable')

with open(Path('training_stats')/'dqn_double_dueling_hyperparams.json', 'w') as f:
    json.dump(res, f, default=json_default)


with open(Path('training_stats')/'dqn_double_dueling_hyperparams2.json') as f:
    dqn_double_dueling_hyper = json.loads(f.read())
    
for param_name in dqn_double_dueling_hyper.keys():
    fig=compare_training_hp(dqn_double_dueling_hyper[param_name], 200, param_name)
    fig.savefig(Path('img')/f'dqn_double_dueling_param_{param_name}.png', dpi=300)


# Parameters that can be changed (hyperparameters)
GAMMA = 0.99          # Gamma parameter for the Bellman equation
BATCH_SIZE = 64       # Batch size
LR = 0.0005           # Learning rate for the optimizer
MEMORY_SIZE = 32000    # Maximum size of the replay buffer
DNN_UPD = 4           # Frequency of updating the main network
DNN_SYNC = 500        # Frequency of synchronizing the target network
N_NEURONS = 128       # Number of neurons in the hidden layers


dqn_hyperparams = {
    'DUELING': True,
    'NOISY_NET': False,
    'DOUBLE': True,
    'N_NEURONS': N_NEURONS,
    'BATCH_SIZE': BATCH_SIZE,
    'GAMMA': GAMMA,
    'LR': LR,
    'MEMORY_SIZE': MEMORY_SIZE,
    'DNN_SYNC': DNN_SYNC,
    'DNN_UPD': DNN_UPD,
}

dqn_res = train_n(dqn_hyperparams, 10)

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 392 Mean Rewards 200.60 Epsilon 0.14		Environment solved in 392 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 548 Mean Rewards 200.90 Epsilon 0.06		Environment solved in 548 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 490 Mean Rewards 211.93 Epsilon 0.09		Environment solved in 490 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 434 Mean Rewards 200.24 Epsilon 0.11		Environment solved in 434 episodes!

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 466 Mean Rewards 201.07 Epsilon 0.10		Environment solved in 466 episodes!


fig = plot_n_training(dqn_res, 200, 'Double Dueling DQN')
fig.savefig(Path('img')/f'dqn_double_dueling_opt_n_train.png', dpi=300)


params = {
    'DUELING': True,
    'NOISY_NET': False,
    'DOUBLE': True,
    'N_NEURONS': N_NEURONS,
    'BATCH_SIZE': BATCH_SIZE,
    'GAMMA': GAMMA,
    'LR': LR,
    'MEMORY_SIZE': MEMORY_SIZE,
    'BATCH_SIZE': BATCH_SIZE,
    'DNN_SYNC': DNN_SYNC,
    'DNN_UPD': DNN_UPD,
}

env = gym.make("LunarLander-v2", render_mode='rgb_array')

DDDQNagent = DQNAgent(env, device='cpu', n_hidden=params['N_NEURONS'], mem_size=params['MEMORY_SIZE'],
                    gamma=params['GAMMA'], lr=params['LR'], noisyNet=params['NOISY_NET'], 
                    dueling=params['DUELING'], double=params['DOUBLE'])

# Train agent
dqn_double_dueling_opt_res = DDDQNagent.train(max_episodes=MAX_EPISODES, 
                            min_episodios=MIN_EPISODES, 
                            batch_size=params['BATCH_SIZE'], 
                            eps_start=EPSILON_START, 
                            eps_end=EPSILON_END,
                            eps_decay=EPSILON_DECAY,
                            dnn_update_frequency=params['DNN_UPD'], 
                            dnn_sync_frequency=params['DNN_SYNC'])

Training:   0%|          | 0/1000 [00:00<?, ?it/s]

Episode 521 Mean Rewards 203.01 Epsilon 0.07		Environment solved in 521 episodes!


total_rewards_dddqn, n_steps_dddqn = play_episode(env, DDDQNagent, save=True)
print(f'Total reward: {total_rewards_dddqn}, Steps: {n_steps_dddqn}')

Total reward: 251.7797835442335, Steps: 325


n_episodes = 1000
total_rewards_dddqn, n_steps_dddqn = play_n_episodes(env, DDDQNagent, n_episodes)

  0%|          | 0/1000 [00:00<?, ?it/s]


plot_results_hist(n_steps_dddqn, total_rewards_dddqn)


test_trained_agent_cmp= pd.DataFrame({'DQN':total_rewards_dqn, 'DDDQN':total_rewards_dddqn}).melt()


fig, ax = plt.subplots(1,2,figsize=(9,5), gridspec_kw={'width_ratios':[2,1]}, sharey=True)
sns.histplot(data=test_trained_agent_cmp, y='value', hue='variable', bins=30, kde=True, ax=ax[0])
ax[0].legend(['DQN', 'Double Dueling DQN'], title='Agent', loc='lower center')
ax[0].set_title('Trained model reward DQN vs. Double Dueling DQN')
ax[0].set_xlabel('Count')
ax[0].set_ylabel('Reward')

# Boxplot
sns.boxplot(x='variable', y='value', hue='variable', data=test_trained_agent_cmp, ax=ax[1])
means = test_trained_agent_cmp.groupby(['variable'])['value'].mean()
ax[1].legend([f'{means["DQN"]:.2f}', f'{means["DDDQN"]:.2f}'], 
             loc='lower center', title='Mean')
leg = ax[1].get_legend()
colors = [color['color'] for color in list(plt.rcParams['axes.prop_cycle'])[:2]]
for i in range(2):
    leg.legendHandles[i].set_color(colors[i])
ax[1].set_xticklabels(['DQN', 'Double Dueling DQN'])
ax[1].set_xlabel('Model')
ax[1].set_ylabel('')
plt.tight_layout()
plt.show()
fig.savefig(Path('img')/f'trained_agent_cmp.png', dpi=300)

Environment¶

DQN Agent¶

Hyperparameter tuning¶

Agente DQN optimo¶

Test de agente entrenado¶

Agente propuesto¶

Double DQN¶

Dueling DQN¶

NoisyNET DQN¶

Double + Dueling DQN¶

Hyperparameters tuning (Double + Dueling DQN)¶

Optimun Double Dueling DQN¶

Prueba agente Double Dueling DQN¶